package com.g2room.myuall.grap.univ;

import java.io.IOException;
import java.net.MalformedURLException;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import com.g2room.driving.grap.GenericGrap;
import com.g2room.driving.util.HtmlUtil;
import com.g2room.driving.util.IpUtil;
import com.g2room.myuall.grap.univ.po.Article;
import com.g2room.myuall.util.DbUtil;
import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException;

public class GrapUnivAtic implements Runnable {
	String imgPath = "nrj";
	public String baseUrl = "http://gaokao.eol.cn";
//	static List<String> titles = GenericMyuall.getTitles();
	static Map<String, String> categoryMap = new HashMap<String, String>();
	static Map<String, String> urlMap = new HashMap<String, String>();
	static {
//		categoryMap.put("高考快讯", "高考");
//		urlMap.put("高考快讯", "http://gaokao.eol.cn/news/");
//		// ---------------- 以下属于报考志愿 
//		// http://gaokao.eol.cn/zhiyuan/tianbao/
//		categoryMap.put("志愿填报", "高考");
//		urlMap.put("志愿填报", "http://gaokao.eol.cn/zhiyuan/tianbao/");
//		// http://gaokao.eol.cn/zhiyuan/zhinan/
//		categoryMap.put("报考指南", "高考");
//		urlMap.put("报考指南", "http://gaokao.eol.cn/zhiyuan/zhinan/");
//		// http://gaokao.eol.cn/zhiyuan/fenshuxian/
//		categoryMap.put("历年分数线", "高考");
//		urlMap.put("历年分数线", "http://gaokao.eol.cn/zhiyuan/fenshuxian/");
//		
//		// ---------------- 以下为大纲解读
//		categoryMap.put("各地动态", "高考");
//		urlMap.put("各地动态", "http://gaokao.eol.cn/gkdg/gedi/");
//		
//		// ---------------- 以下为专业资讯
//		// http://zhuanye.eol.cn/jiedu/
//		categoryMap.put("专业解读", "高考");
//		urlMap.put("专业解读", "http://zhuanye.eol.cn/jiedu/");
//		// ---------------- 以下为自招动态
//		// http://gaokao.eol.cn/zizhu/news/
//		categoryMap.put("自招动态", "高考");
//		urlMap.put("自招动态", "http://gaokao.eol.cn/zizhu/news/");
//		// http://gaokao.eol.cn/zizhu/zzzxpl/
//		categoryMap.put("分析评论", "高考");
//		urlMap.put("分析评论", "http://gaokao.eol.cn/zizhu/zzzxpl/");
//		
//		
//		
//		// ---------------- 以下为报名信息
//		// http://gaokao.eol.cn/gkbm/bmxx/
//		categoryMap.put("报名信息", "高考");
//		urlMap.put("报名信息", "http://gaokao.eol.cn/gkbm/bmxx/");
//		// ---------------- 以下为高职
//		// http://gaokao.eol.cn/gaozhi/gzkx/
//		categoryMap.put("招生快讯", "高考");
//		urlMap.put("招生快讯", "http://gaokao.eol.cn/gaozhi/gzkx/");
//		categoryMap.put("报考指导", "高考");
//		urlMap.put("报考指导", "http://gaokao.eol.cn/gaozhi/cjwt/");
//		categoryMap.put("院校动态", "高考");
//		urlMap.put("院校动态", "http://gaokao.eol.cn/gaozhi/zsxx/");
//		
//		
//		
//		
//		// ---------------- 以下为保送生
//		categoryMap.put("保送动态", "高考");
//		urlMap.put("保送动态", "http://gaokao.eol.cn/baosong/news/");
//		categoryMap.put("各校政策", "高考");
//		urlMap.put("各校政策", "http://gaokao.eol.cn/baosong/zhengce/");
//		
//		// ---------------- 以下为特长生
//		categoryMap.put("艺术特长生", "高考");
//		urlMap.put("艺术特长生", "http://gaokao.eol.cn/ystc/ystcs/");
//		categoryMap.put("体育特长生", "高考");
//		urlMap.put("体育特长生", "http://gaokao.eol.cn/ystc/tytcs/");
//		
//		// ---------------- 以下为国防生
//		categoryMap.put("国防生", "高考");
//		urlMap.put("国防生", "http://gaokao.eol.cn/junshi/guofang/");
//		categoryMap.put("军校生", "高考");
//		urlMap.put("军校生", "http://gaokao.eol.cn/junshi/junxiao/");
//		categoryMap.put("招飞快讯", "高考");
//		urlMap.put("招飞快讯", "http://gaokao.eol.cn/zhaofei/fxxy/");
//		
//		
//		// ---------------- 以下为小语种
//		categoryMap.put("小语种", "高考");
//		urlMap.put("小语种", "http://gaokao.eol.cn/xiaoyz/zxdt/");
//		
//		// ---------------- 以下为艺考
//		categoryMap.put("艺考快讯", "高考");
//		urlMap.put("艺考快讯", "http://yikao.eol.cn/xinwen/");
//		categoryMap.put("明星名人", "高考");
//		urlMap.put("明星名人", "http://yikao.eol.cn/mingxing/");
//		
//		// ------------- 高校集合
//		// http://gaokao.eol.cn/daxue/zixun/
//		categoryMap.put("高校资讯", "高考");
//		urlMap.put("高校资讯", "http://gaokao.eol.cn/daxue/zixun/");
//		// http://gaokao.eol.cn/daxue/fangtan/
//		categoryMap.put("招生访谈", "高考");
//		urlMap.put("招生访谈", "http://gaokao.eol.cn/daxue/fangtan/");
//		// http://gaokao.eol.cn/daxue/mingdan/
//		categoryMap.put("高校名单", "高考");
//		urlMap.put("高校名单", "http://gaokao.eol.cn/daxue/mingdan/");
		
		// ------------- 高考政策
		categoryMap.put("报考政策", "高考");
		urlMap.put("报考政策", "http://gaokao.eol.cn/zhengce/baokao/");
		categoryMap.put("录取政策", "高考");
		urlMap.put("录取政策", "http://gaokao.eol.cn/zhengce/luqu/");
		categoryMap.put("加分政策", "高考");
		urlMap.put("加分政策", "http://gaokao.eol.cn/zhengce/jfzc/jiafen/");
		categoryMap.put("体检政策", "高考");
		urlMap.put("体检政策", "http://gaokao.eol.cn/zhengce/tijian/");
	}
	private String type;
	private String url;
	
	GrapUnivAtic(String type, String url) {
		this.type = type;
		this.url = url;
	}
	public static void main(String[] args) throws Exception{
		for(String type : urlMap.keySet()) {
			GrapUnivAtic gkd = new GrapUnivAtic(type, urlMap.get(type));
			Thread t = new  Thread(gkd);
			t.start();
		}
		
//		for(String url : list) {
//			for(String type : urlMap.keySet()) {
//				if(url.startsWith(urlMap.get(type))) {
//					System.out.println(url);
//					GrapUnivAtic gkd = new GrapUnivAtic(type, urlMap.get(type));
//					gkd.getPage(url);
//				}
//			}
//		}
		
//		Map<String, Integer> errMap = getErrs();
//		for(String url : errMap.keySet()) {
//			int urlType = errMap.get(url);
//			for(String type : urlMap.keySet()) {
//				System.out.println(url + "==" + urlMap.get(type));
//				if(url.startsWith(urlMap.get(type))) {
//					GrapUnivAtic gkd = new GrapUnivAtic(type, urlMap.get(type));
//					switch (urlType) {
//					case 2:
//						gkd.getPage(url);
//						break;
//					}
//				}
//			}
//		}
    }
	
	/**
	 * 获取分页列表
	 * @param url
	 * @throws FailingHttpStatusCodeException
	 * @throws MalformedURLException
	 * @throws IOException
	 */
	public void getNav() {
		List<String> pages = new ArrayList<String>();
		// 获取分页信息
		Document doc = null;
		try {
			doc = GenericGrap.getDocByIp(url);
		} catch (IOException e) {
			System.out.println(url);
			e.printStackTrace();
			insert2ErrDb(url, 1);
		}
		if(doc == null) return;
		
		int pageNum = getPageUrl(doc.getElementById("pagenav").html());
		// 首页单独处理
		pages.add(this.url);
		for(int i= 1; i<pageNum; i++ ) {
			pages.add(this.url + "/index_" + i + ".shtml");
		}
        System.out.println(this.url + " 本次抓取页数：" + pages.size());
		
        for(String url : pages) {
        	getPage(url);
        }
	}
	
	private Integer getPageUrl(String url) {
		url = url.substring(url.indexOf("_PAGE_COUNT=") + 13);
		String page = url.substring(0, Math.min(2, url.indexOf("\"")));
		return Integer.valueOf(page);
	}
	
	/**
	 * 获取分页数据
	 * @param url
	 * @throws FailingHttpStatusCodeException
	 * @throws MalformedURLException
	 * @throws IOException
	 */
	public boolean getPage(String url) {
		Document doc = null;
		try {
			doc = GenericGrap.getDocByIp(url);
		} catch (IOException e) {
//			e.printStackTrace();
			System.out.println(url);
			insert2ErrDb(url, 2);
		}
		if(doc == null) return false;
		
		Elements liEls = doc.getElementsByAttributeValue("class", "page_left").first().getElementsByTag("li");
		List<Article> artis = new ArrayList<Article>();
		for(Element el : liEls) {
			try {
				Element titleEl = el.getElementsByTag("a").first();
				String href = titleEl.attr("href");
				
				Article arti = new Article();
				arti.setTitle(titleEl.text());
				arti.setPublishTime(el.getElementsByTag("span").first().text());
				String sourceUrl = "";
				int state = 0;
				// 特殊处理
				if(href.startsWith("http")) {
					sourceUrl = href;
				} else if(href.startsWith("./")) {
					sourceUrl = this.url + href.replace("./", "");
				} else {
					state = 9;
					sourceUrl = baseUrl + "/" + href.replace("../", "");
				}
				arti.setSourceUrl(sourceUrl);
				arti.setTypeName(this.type);
				// 通过状态区分类型，本目录=0；其他目录为9
				arti.setState(state);
				artis.add(arti);
			} catch (Exception e) {
				e.printStackTrace();
			}
		}
		
        for(Article arti : artis) {
        	try {
				Thread.sleep(IpUtil.genRandomNum() * 10);
			} catch (InterruptedException e) {
				e.printStackTrace();
			}
        	getDetail(arti);
        }
        return true;
	}
	
	public void getDetail(Article arti) {

		Document doc = null;
		try {
			doc = GenericGrap.getDocByIp(arti.getSourceUrl());
		} catch (IOException e) {
			System.out.println(arti.getSourceUrl());
//			e.printStackTrace();
			insert2ErrDb(arti.getSourceUrl(), 3);
		}
		if(doc == null) return;
		
		try {
			// 获取分页信息
			int pageNum = 0;
			try {
				Element pel = doc.getElementById("pagenav");
				if(null == pel) return;
				
				pageNum = getPageUrl(pel.html());
				arti.setPageNum(pageNum);
			} catch (Exception e) {
				System.out.println("242==="+arti.getSourceUrl());
//				e.printStackTrace();
			}
			
			String content = doc.getElementsByAttributeValue("class", "TRS_Editor").html();
			// 标记获取标签
			boolean flag = false;
			// 如果为空则尝试从mcontent 获取
			if(null == content || content.length() < 5) {
				content = doc.getElementById("mcontent").html();
				flag = true;
			}
			if(pageNum > 1) {
				for(int i=1; i< pageNum; i++) {
					String href = arti.getSourceUrl().replace(".shtml", "_"+i+".shtml");
					Document pageDoc = null;
					try {
						pageDoc = GenericGrap.getDocByIp(href);
					} catch (Exception e) {
//						e.printStackTrace();
						insert2ErrDb(href, 4);
						return;
					}
					if(pageDoc == null) continue;
					content += "\n"+pageDoc.getElementsByAttributeValue("class", "TRS_Editor").html();
					// 如果为空则尝试从mcontent 获取
					if(flag) {
						content += "\n"+pageDoc.getElementById("mcontent").html();
					}
				}
			}
			content = GenericMyuall.handleSpace(HtmlUtil.delAllTag(content));
			
			arti.setContent(content);
			// 获取发布时间
			insert2Db(arti);

//			insert2GrapedDb(arti.getSourceUrl(), 1);
        } catch (Exception e) {
        	e.printStackTrace();
			insert2ErrDb(arti.getSourceUrl(), 5);
        }
	}
	

	public static void insert2Db(Article article) {
		try {
			PreparedStatement ps = DbUtil.getConn().prepareStatement("insert into ncee_acticle(title,content,publish_time,type_name,source,source_url,state,category_name,page_num) values(?,?,?,?,?,?,?,?,?)");
			ps.setString(1, article.getTitle());
			ps.setString(2, article.getContent());
			ps.setString(3, article.getPublishTime());
			ps.setString(4, article.getTypeName());
			ps.setString(5, article.getSource());
			ps.setString(6, article.getSourceUrl());
			ps.setInt(7, article.getState());
			ps.setString(8, article.getCategoryName());
			ps.setInt(9, article.getPageNum());
			ps.execute();
			
		} catch(Exception e) {
			e.printStackTrace();
		}
	}
	
	public void insert2ErrDb(String url, int type) {
		try {
			PreparedStatement grapPs = DbUtil.getConn().prepareStatement("INSERT INTO `univ_err` (`url`,type) "
					+ "values(?,?)");
			grapPs.setString(1, url);
			grapPs.setInt(2, type);
			grapPs.execute();
		} catch(Exception e) {
			e.printStackTrace();
		}
	}
	
	public void insert2GrapedDb(String url, int type) {
		try {
			PreparedStatement grapPs = DbUtil.getConn().prepareStatement("INSERT INTO `univ_graped_arti` (`url`,type) "
					+ "values(?,?)");
			grapPs.setString(1, url);
			grapPs.setInt(2, type);
			grapPs.execute();
		} catch(Exception e) {
			e.printStackTrace();
		}
	}
	
	

	public void updateContent(String content, int id) {
		try {
			PreparedStatement grapPs = DbUtil.getConn().prepareStatement("update `ncee_acticle` set content=? where id=? ");
			grapPs.setString(1, content);
			grapPs.setInt(2, id);
			grapPs.execute();
		} catch(Exception e) {
			e.printStackTrace();
		}
	}
	public static Map<String, Integer> getNullContent() {
		Map<String, Integer> errs = new HashMap<String, Integer>();
		try {
			Connection connection = DbUtil.getConn();
			Statement statement = connection.createStatement();
			String sql = "select id,source_url from ncee_acticle where content is null or content = ''";
			ResultSet rs = statement.executeQuery(sql);
			while(rs.next()) {
				errs.put(rs.getString("source_url"), rs.getInt("id"));
			}
			rs.close();
			statement.close();
		} catch(Exception e) {
			e.printStackTrace();
		}
		return errs;
	}
	public static Map<String, Integer> getErrs() {
		Map<String, Integer> errMap = new HashMap<String, Integer>();
		try {
			Connection connection = DbUtil.getConn();
			Statement statement = connection.createStatement();
			String sql = "select url,type from univ_err where type=2";
			ResultSet rs = statement.executeQuery(sql);
			while(rs.next()) {
				errMap.put(rs.getString("url"), rs.getInt("type"));
			}
			rs.close();
			statement.close();
		} catch(Exception e) {
			e.printStackTrace();
		}
		return errMap;
	}
	

	public void getDetail(String url, int id) {
		Document doc = null;
		try {
			doc = GenericGrap.getDocByIp(url);
		} catch (IOException e) {
			System.out.println(url);
//			e.printStackTrace();
			insert2ErrDb(url, 9);
		}
		if(doc == null) return;
		
		try {
			// 获取分页信息
			int pageNum = 0;
			try {
				pageNum = getPageUrl(doc.getElementById("pagenav").html());
			} catch (Exception e) {
				System.out.println("111==="+url);
//				e.printStackTrace();
				throw e;
			}
			
			String content = doc.getElementsByAttributeValue("class", "TRS_Editor").html();
			// 标记获取标签
			boolean flag = false;
			// 如果为空则尝试从mcontent 获取
			if(null == content || content.length() < 5) {
				content = doc.getElementById("mcontent").html();
				flag = true;
			}
			if(pageNum > 1) {
				for(int i=1; i< pageNum; i++) {
					String href = url.replace(".shtml", "_"+i+".shtml");
					Document pageDoc = null;
					try {
						pageDoc = GenericGrap.getDocByIp(href);
					} catch (Exception e) {
//						e.printStackTrace();
						insert2ErrDb(href, 4);
					}
					if(doc == null) continue;
					content += "\n"+pageDoc.getElementsByAttributeValue("class", "TRS_Editor").html();
					// 如果为空则尝试从mcontent 获取
					if(flag) {
						content += "\n"+pageDoc.getElementById("mcontent").html();
					}
				}
			}
			content = GenericMyuall.handleSpace(HtmlUtil.delAllTag(content));
			
			// 获取发布时间
			updateContent(content, id);
        } catch (Exception e) {
			System.out.println(url);
//        	e.printStackTrace();
			insert2ErrDb(url, 5);
        }
	}

	@Override
	public void run() {
		getNav();
	}
}
